# Scrapy settings for newfk project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'newfk'

SPIDER_MODULES = ['newfk.spiders']
NEWSPIDER_MODULE = 'newfk.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'newfk (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 130

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0
# The download delay setting will honor only one of:
CONCURRENT_REQUESTS_PER_DOMAIN = 130
CONCURRENT_REQUESTS_PER_IP = 130

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
import time



import random


def ua():
    ua_pool = [

        "Mozilla/5.0 (Windows NT 6.2;en-US) AppleWebKit/537.32.36 (KHTML, live Gecko) Chrome/59.0.3047.101 Safari/537.32",

        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36",

        "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.5(0x18000528) NetType/4G Language/zh_CN",

        "Mozilla/5.0 (Linux; Android 11; M2012K11AC Build/RKQ1.200826.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.120 MQQBrowser/6.2 TBS/045613 Mobile Safari/537.36 MMWEBID/7803 MicroMessenger/8.0.3.1880(0x2800033D) Process/tools WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64",

        "Mozilla/5.0 (Linux; Android 9; Mi Note 3 Build/PKQ1.181007.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.120 MQQBrowser/6.2 TBS/045609 Mobile Safari/537.36 MMWEBID/4799 MicroMessenger/8.0.2.1860(0x2800023D) Process/tools WeChat/arm64 Weixin NetType/4G Language/zh_CN ABI/arm64",

        "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.5(0x18000528) NetType/4G Language/zh_CN",

        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",

        "Mozilla/5.0 (Linux; Android 11; V1824A Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.120 MQQBrowser/6.2 TBS/045611 Mobile Safari/537.36 MMWEBID/8305 MicroMessenger/8.0.2.1860(0x2800023D) Process/tools WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64",

        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36",

        "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.3(0x1800032c) NetType/WIFI Language/zh_CN",

        "Mozilla/5.0 (Linux; U; Android 10; zh-cn; MI 9 Build/QKQ1.190825.002) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.3.12",
    ]
    return ua_pool[random.randint(1, 10)]


chrome_options = Options()
# chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
# chrome_options.add_experimental_option("useAutomationExtension", False)

chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-sandbox")
# rizhi=chrome_options.add_argument('--log-level=3')
chrome_options.add_argument("--no-sandbox")
browser = webdriver.Chrome(options=chrome_options)
# browser = webdriver.Chrome()
browser.get('https://www.4kmiles.com/console/dashboard')
time.sleep(10)
# 输入账户密码
browser.find_elements_by_class_name('el-input__inner')[0].click()
browser.find_elements_by_class_name('el-input__inner')[0].send_keys('nicolechow1993@163.com')
browser.find_elements_by_class_name('el-input__inner')[1].click()
browser.find_elements_by_class_name('el-input__inner')[1].send_keys('nicolez123')
browser.find_element_by_xpath('//button[@class="el-button login-btn el-button--primary"]').send_keys(Keys.ENTER)
time.sleep(10)
cookie = [item["name"] + "=" + item["value"] for item in browser.get_cookies()]
print(cookie)
list_canshu = []
for i in cookie:
    if 'acc_t' in i:
        canshu = 'Bearer ' + i[6:]
        list_canshu.append(canshu)
        print(canshu)

browser.quit()


DEFAULT_REQUEST_HEADERS = {
    'referer': 'https://www.4kmiles.com/console/dashboard',
    'User-Agent': ua(),

    'authorization': f'{list_canshu[0]}'
}




# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'newfk.middlewares.NewfkSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'newfk.middlewares.NewfkDownloaderMiddleware': 543,
}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   # 'newfk.pipelines.LvyouPipeline': 300,
    'newfk.pipelines.MySQLPipeline': 300
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'


# import datetime
# LOG_LEVEL = 'DEBUG'
# to_day = datetime.datetime.now()
# log_file_path = 'scrapy_{}_{}_{}_{}.log'.format(to_day.year, to_day.month, to_day.day,to_day.hour)
# LOG_FILE = log_file_path


